﻿using System;
using System.Text;
using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

namespace Tenth_Assignment {
    class Crawler {
        private int count;
        private Hashtable urls;

        public int Count { get => count; }
        public Hashtable Urls { get => urls; }

        public async Task<bool> CrawlAsync(string startUrl, int maxNum) {
            return await Task.Run(() => Crawl(startUrl, maxNum));
        }

        public bool Crawl(string startUrl, int maxNum) {
            if (urls == null) {
                try {
                    if (!new Uri(startUrl).IsAbsoluteUri) {
                        return false;
                    }
                } catch (UriFormatException) {
                    return false;
                }
                count = 0;
                urls = new Hashtable {
                        { startUrl, false }
                    };
            }
            Parallel.For(1, 5, index => ConcurrentCrawl(maxNum));
            return true;
        }

        private void ConcurrentCrawl(int maxNum) {
            while (true) {
                string current = null;
                lock (urls) {
                    foreach (string url in urls.Keys) {
                        if ((bool)urls[url]) {
                            continue;
                        }
                        current = url;
                    }
                    if (count >= maxNum || current == null) {
                        break;
                    }
                    Console.WriteLine(current);
                    urls[current] = true;
                    count++;
                }
                string html = DownLoad(current);
                Parse(current, html);
            }
        }

        private string DownLoad(string url) {
            try {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                File.WriteAllText(count.ToString() + (Path.HasExtension(url) ? Path.GetExtension(url) : ".html"), html, Encoding.UTF8);
                return html;
            } catch (Exception e) {
                Console.WriteLine(e.Message);
                return "";
            }
        }

        private void Parse(string rootUrl, string html) {
            Uri baseUri = new Uri(rootUrl);
            MatchCollection matches = new Regex(@"(href|HREF)\s*=\s*[""](?<Url>[^""]+[.](html|aspx|jsp))[""?#]").Matches(html);
            foreach (Match match in matches) {
                try {
                    Uri uri = new Uri(baseUri, match.Groups["Url"].Value);
                    if (uri.Host == baseUri.Host) {
                        lock (urls) {
                            if (urls[uri.AbsoluteUri] == null) {
                                urls[uri.AbsoluteUri] = false;
                            }
                        }
                    }
                } catch (UriFormatException) { }
            }
        }
    }
}
